//package com.bonc.extractor;
//
//import java.io.File;
//import java.io.IOException;
//import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.HashSet;
//import java.util.List;
//import java.util.Map;
//import java.util.Set;
//
//import javax.management.loading.PrivateClassLoader;
//
//import org.apache.commons.io.FileUtils;
//
//import com.apporiented.algorithm.clustering.Cluster;
//import com.bonc.text.entity.recognition.RecResult;
//import com.bonc.text.service.impl.EntityRecognitionServiceImpl;
//import com.bonc.utilities.EventUtility;
//import com.bonc.vectorspace.model.Corpus;
//import com.bonc.vectorspace.model.Document;
//import com.bonc.vectorspace.model.VectorSpaceModel;
//
//import cc.mallet.classify.Classifier;
//import cc.mallet.classify.ClassifierTrainer;
//import cc.mallet.classify.MaxEntTrainer;
//import cc.mallet.pipe.SerialPipes.Predicate;
//import cc.mallet.types.Alphabet;
//import cc.mallet.types.FeatureVector;
//import cc.mallet.types.Instance;
//import cc.mallet.types.InstanceList;
//import cc.mallet.types.Label;
//import cc.mallet.types.LabelAlphabet;
//import cc.mallet.types.Labeling;
//
///**
// * @author donggui@bonc.com.cn
// * @version 2016 2016年6月23日 上午11:43:17
// */
//public class EventExtractTester {
//	
//	private final double AalphaParam = 0.6;
//	
//	private final double Threshold = 0.5;
//	
//	private double DistanceThreshold = 0.60;
//	
//
//
//	private String text;
//	
//	private ArrayList<Document> documents = new ArrayList<Document>();
//	
//	private Set<String> featureTerms;
//	
//	private VectorSpaceModel vectorSpace;
//	
//	public static Map<String, String> stopwords = EventUtility.loadStopwords();
//	
//	public EventExtractTester(){
//		
//	}
//
//	public EventExtractTester(String text){
//		this.setText(text.trim());
//		String[] sentences = sentenceSegment();
//		int validCount = 0;
//		for(int i=0; i< sentences.length; i++){
//			String sent = sentences[i];
//			if(sent!=null && !"".equals(sent.trim())){ 
//				Document document = new Document(String.valueOf(validCount), sentences[i], true, stopwords);
//				documents.add(document);
//				validCount++;
//			}
//		}
//	}
//	
//	public double getDistanceThreshold() {
//		return DistanceThreshold;
//	}
//
//	public void setDistanceThreshold(double distanceThreshold) {
//		DistanceThreshold = distanceThreshold;
//	}
//	
//	private String[] sentenceSegment(){
//		if(text!=null){
//			return text.split("\n|。|？|！");
//		}
//		return null;
//	}
//
//	public String getText() {
//		return text;
//	}
//
//	public void setText(String text) {
//		this.text = text;
//	}
//	
//	public Classifier getTrainedClassifier() throws IOException{
//		Map<String, String> stopwords = EventUtility.loadStopwords();
//		
//		String sPath = "D:\\workspace\\extractor\\data\\events\\";
//		List<String> fileNames = MaxentChiTester.getFileList(sPath);
//		
//		ArrayList<Document> documents = new ArrayList<Document>();
//        ArrayList<String> targetValue = new ArrayList<String>();
//        
//		for(String fname:fileNames){
//			int k = 1;
//			File file = new File(sPath+fname);
//			String content = FileUtils.readFileToString(file);
//			String[] events = content.split("\n|。|？|！");
//			for(String event: events){
//				String id = fname + k;
//				System.out.println(id+"=="+event);
//				if(event!=null && !"".equals(event.trim())){
//					Document document = new Document(id,event,true,stopwords);
//					documents.add(document);
//					targetValue.add(fname.replaceAll(".txt", ""));
//					k++;
//				}				
//			}
//		}		
//		
//		Corpus corpus = new Corpus(documents);
//		VectorSpaceModel vectorSpace = new VectorSpaceModel(corpus);		
//		int row = documents.size();
////		Set<String> terms = corpus.getInvertedIndex().keySet();	
//		this.featureTerms = corpus.getInvertedIndex().keySet();	
//
//		//maxent
//        Alphabet featureAlphabet = new Alphabet();//特征词典
//        LabelAlphabet targetAlphabet = new LabelAlphabet();//类标词典
//        targetAlphabet.lookupIndex("pos");
//        targetAlphabet.lookupIndex("neg");
//        targetAlphabet.stopGrowth();
////      featureAlphabet.lookupIndex("f1");
////      featureAlphabet.lookupIndex("f2");
//        for (String term : featureTerms) {
//        	featureAlphabet.lookupIndex(term);	       	
//        }
//
//        InstanceList allInstances = new InstanceList (featureAlphabet,targetAlphabet);//实例集对象
//        
//        int featuresize = featureTerms.size();
//        int i = 0;
//		for (Document document : corpus.getDocuments()) {
////			System.out.print("document "+(document.getFileName()+"===="));
//			HashMap<String, Double> weights = vectorSpace.getTfIdfWeights().get(document);
//			double[] featureValues1 = new double[featuresize];
//			int j = 0;
//			for (String term : featureTerms) {				
//				Double weight = weights.get(term);
//				if(weight !=null ){
//					featureValues1[j] = weight.doubleValue();
//				}else{
//					featureValues1[j] = 0.0;
//				}
//				j++;				
//			}
//			System.out.println();			
//            
//            FeatureVector featureVector = new FeatureVector(featureAlphabet,featureTerms.toArray(new String[featuresize]),featureValues1);//change list to array           
//            Instance instance = new Instance (featureVector,targetAlphabet.lookupLabel(targetValue.get(i)), document.getFileName(),null);
//            
//            i++;
//            allInstances.add(instance);
//		}
//		
//		double Gaussian_Variance = 1.0;
//		ClassifierTrainer trainer = new MaxEntTrainer(Gaussian_Variance);
//		
//		InstanceList trainingInstances = allInstances.subList(0, row-2);
//		Instance testinstance = allInstances.get(row-1);
//		Classifier maxentclassifier = trainer.train(trainingInstances); 
//		return maxentclassifier;
//	}
//
//	public List<String> getPredicts() throws IOException{
//		if(documents!=null && documents.size()>0){
//			ArrayList<String> targetValue = new ArrayList<String>();
//			
//			Classifier classifier = this.getTrainedClassifier();
//	        Alphabet featureAlphabet = classifier.getAlphabet();//特征词典
//	        LabelAlphabet targetAlphabet = classifier.getLabelAlphabet();//类标词典
//	        
////	        InstanceList instances = new InstanceList (featureAlphabet,targetAlphabet);//实例集对象
//	        
//			Corpus corpus = new Corpus(documents);
//			this.vectorSpace = new VectorSpaceModel(corpus,featureTerms);		
////			int row = documents.size();
//			int featuresize = featureTerms.size();
//			
//			for (Document document : corpus.getDocuments()) {
//				System.out.println("document "+(document.getFileName()+"===="+document.getContent()));
//				HashMap<String, Double> weights = vectorSpace.getTfIdfWeights().get(document);
//				double[] featureValues1 = new double[featuresize];
//				int j = 0;
//				for (String term : featureTerms) {				
//					Double weight = weights.get(term);
//					if(weight !=null ){
//						featureValues1[j] = weight.doubleValue();
//					}else{
//						featureValues1[j] = 0.0;
//					}
//					j++;				
//				}
//	            
//	            FeatureVector featureVector = new FeatureVector(featureAlphabet,featureTerms.toArray(new String[featuresize]),featureValues1);//change list to array
//	            
//	            Instance testInstance = new Instance (featureVector,null, document.getFileName(),null);
//		        Labeling labeling = classifier.classify(testInstance).getLabeling();
//		        Label bestLabel = labeling.getBestLabel();
//		        double bestValue = labeling.getBestValue();
//		        if("pos" == (String)bestLabel.getEntry() && bestValue >= this.Threshold){
//		        	targetValue.add((String)bestLabel.getEntry());
//		        	System.out.println(labeling.toString());
////		        	System.out.println((String)bestLabel.getEntry() + ':'+bestLabel.getBestValue());
//		        }else{
//		        	targetValue.add("neg");
//		        }
//			}
//			if(targetValue !=null){
//				return targetValue;
//			}			
//		}
//		return null;
//	}
//	
//	public Cluster getCluster(List<Integer> indices){
//		//select pos documents
////		int row = documents.size();
//		List<Document> newDocuments = new ArrayList<Document>();
//		for(int index:indices){
//			newDocuments.add(documents.get(index));
//		}
//		int row = indices.size();
//		double[][] distances = new double[row][row];
//		for (int i = 0; i < row; i++){
//			for (int j = i+1; j< row; j++){
//				int row_idx = indices.get(i);
//				int col_idx = indices.get(j);
//				Document doci = documents.get(row_idx);
//				Document docj = documents.get(col_idx);
//				double similarity = vectorSpace.jaccardSimilarity(doci, docj);
//				distances[i][j] = AalphaParam * (1-similarity) + (1-AalphaParam)*(j-i)/row;
//				distances[j][i] = distances[i][j];
//				distances[i][i] = 0;
//			}
//		}		
//		for(int i = 0; i < row; ++i){
//			for(int j = 0; j < row; ++j){
//				System.out.print((1-distances[i][j]) + "\t");
//			}			
//			System.out.println();
//		}		
//		System.out.println();
//		VectorAndClusterChiTester2 clusterChiTester2 = new VectorAndClusterChiTester2();
//		Cluster cluster = clusterChiTester2.createSampleCluster(newDocuments,distances);
//		if(cluster !=null){
//			return cluster;
//		}		
//		return null;		
//	}
//	
//	public String firstCluster(Cluster clu){
//		String result = "";
//		if(clu !=null & clu.getChildren() !=null){
//			List<Cluster> children = clu.getChildren();
//			for(int j =0; j<children.size(); j++){
//				Cluster child = children.get(j);
////				System.out.print(child.getName()+"==");
//				if(child.isLeaf()){
//					String childName = child.getName();
//					Document doc = documents.get(Integer.valueOf(childName));
////					System.out.print(doc.getContent());
//					result += childName + "==" + doc.getContent() + "==";
//				}
//				if(children.get(j).getChildren()!=null){
//					result += firstCluster(child) + '\n';
//				}	
//			}
//		}
//		return result;
//	}
//	
//	public Set<Cluster> selectSubClusters(Cluster clu){
////		String result = "";
//		Set<Cluster> clusters = new HashSet<Cluster>();
//		if(clu !=null & clu.getChildren() !=null){
//			List<Cluster> children = clu.getChildren();
//			for(int j =0; j<children.size(); j++){
//				Cluster child = children.get(j);
//				if(!child.isLeaf()){
//					String childName = child.getName();
//					double distanceValue = child.getDistanceValue();
//					if(distanceValue <= DistanceThreshold){
//						clusters.add(child);
//						System.out.println("adding cluster=="+child.getName()+":"+distanceValue);
//					}else{
//						clusters.addAll(selectSubClusters(child));
//					}
////					System.out.print(doc.getContent());
//				}	
//			}
//		}
//		return clusters;
//	}
//	
//	private Set<Document> getEvents(Cluster cluster){		
//		//HashMap<String, Set<Document>> clus = new HashMap<String, Set<Document>>();
//
//		Set<Document> subdocs = new HashSet<Document>();
//		if(!cluster.isLeaf()){
//			subdocs.addAll(getEvents(cluster));
//		}else{
//			String leafId = cluster.getName();
//			Document document = documents.get(Integer.valueOf(leafId));
//			subdocs.add(document);
//		}
//
//		return subdocs;
//	}
//	
//	public HashMap<String, Set<Document>> selectEvents(Set<Cluster> clusters){
//		HashMap<String, Set<Document>> events = new HashMap<String, Set<Document>>();
//		for(Cluster clu:clusters){
//			events.put(clu.getName(), getEvents(clu));
//		}
//		return events;
//	}
//	
//	public static void main(String[] args) throws IOException {
//		String path = "D:\\workspace\\extractor\\data\\demo_e1\\";
//		String fname = "event8.txt";
//		String fpath = path + fname;
//		File file = new File(fpath);
//		String text = FileUtils.readFileToString(file);
//		EventExtractTester tester = new EventExtractTester(text);
//		List<String> targetValues = tester.getPredicts();
//		
//		List<Integer> event_index = new ArrayList<Integer>();
//		for(int k = 0; k<targetValues.size(); k++){
//			if(targetValues.get(k) == "pos"){
//				event_index.add(k);
//			}
//		}
//		
//		System.out.println(tester.firstCluster(tester.getCluster(event_index)));
//		
////		Cluster parentCluster = tester.getCluster(event_index);
////		Set<Cluster> clusters =tester.selectSubClusters(parentCluster);
////		while(clusters==null || clusters.size()<1){
////			tester.setDistanceThreshold(tester.getDistanceThreshold() + 0.03);
////			clusters =tester.selectSubClusters(parentCluster);
////		}
//		
////		EntityRecognitionServiceImpl srv = new EntityRecognitionServiceImpl();
////		HashMap<String, Set<Document>> eventmap = tester.selectEvents(clusters);
////		for(String key:eventmap.keySet()){
////			Set<Document> docs = eventmap.get(key);
////			String content = "";
////			System.out.println("=="+key+"==");
////			for(Document doc:docs){
////				content += doc.getContent();
////			}
////			List<RecResult> results = srv.getNameEntityList(content);
////			for(RecResult res:results){
////				System.out.println(res.getWord());
////			}
////		}
//	}
//}
